

<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<head>
  <meta charset="utf-8" />
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
  
  <title>mindspore.dataset.AGNewsDataset &mdash; MindSpore master documentation</title>
  

  
  <link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />

  
  

  
  

  

  
  <!--[if lt IE 9]>
    <script src="../../_static/js/html5shiv.min.js"></script>
  <![endif]-->
  
    
      <script type="text/javascript" id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script>
        <script src="../../_static/jquery.js"></script>
        <script src="../../_static/underscore.js"></script>
        <script src="../../_static/doctools.js"></script>
        <script src="../../_static/language_data.js"></script>
        <script async="async" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/latest.js?config=TeX-AMS-MML_HTMLorMML"></script>
    
    <script type="text/javascript" src="../../_static/js/theme.js"></script>

    
    <link rel="index" title="Index" href="../../genindex.html" />
    <link rel="search" title="Search" href="../../search.html" />
    <link rel="next" title="mindspore.dataset.AmazonReviewDataset" href="mindspore.dataset.AmazonReviewDataset.html" />
    <link rel="prev" title="mindspore.dataset.WIDERFaceDataset" href="mindspore.dataset.WIDERFaceDataset.html" /> 
</head>

<body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search" >
          

          
            <a href="../../index.html" class="icon icon-home"> MindSpore
          

          
          </a>

          
            
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        
        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption"><span class="caption-text">MindSpore Python API</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../mindspore.html">mindspore</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mindspore.common.initializer.html">mindspore.common.initializer</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mindspore.communication.html">mindspore.communication</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mindspore.compression.html">mindspore.compression</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mindspore.context.html">mindspore.context</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="../mindspore.dataset.html">mindspore.dataset</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="../mindspore.dataset.html#id1">视觉</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="../mindspore.dataset.html#id2">文本</a><ul class="current">
<li class="toctree-l3 current"><a class="current reference internal" href="#">mindspore.dataset.AGNewsDataset</a></li>
<li class="toctree-l3"><a class="reference internal" href="mindspore.dataset.AmazonReviewDataset.html">mindspore.dataset.AmazonReviewDataset</a></li>
<li class="toctree-l3"><a class="reference internal" href="mindspore.dataset.CLUEDataset.html">mindspore.dataset.CLUEDataset</a></li>
<li class="toctree-l3"><a class="reference internal" href="mindspore.dataset.CoNLL2000Dataset.html">mindspore.dataset.CoNLL2000Dataset</a></li>
<li class="toctree-l3"><a class="reference internal" href="mindspore.dataset.CSVDataset.html">mindspore.dataset.CSVDataset</a></li>
<li class="toctree-l3"><a class="reference internal" href="mindspore.dataset.DBpediaDataset.html">mindspore.dataset.DBpediaDataset</a></li>
<li class="toctree-l3"><a class="reference internal" href="mindspore.dataset.EnWik9Dataset.html">mindspore.dataset.EnWik9Dataset</a></li>
<li class="toctree-l3"><a class="reference internal" href="mindspore.dataset.IMDBDataset.html">mindspore.dataset.IMDBDataset</a></li>
<li class="toctree-l3"><a class="reference internal" href="mindspore.dataset.IWSLT2016Dataset.html">mindspore.dataset.IWSLT2016Dataset</a></li>
<li class="toctree-l3"><a class="reference internal" href="mindspore.dataset.IWSLT2017Dataset.html">mindspore.dataset.IWSLT2017Dataset</a></li>
<li class="toctree-l3"><a class="reference internal" href="mindspore.dataset.PennTreebankDataset.html">mindspore.dataset.PennTreebankDataset</a></li>
<li class="toctree-l3"><a class="reference internal" href="mindspore.dataset.SogouNewsDataset.html">mindspore.dataset.SogouNewsDataset</a></li>
<li class="toctree-l3"><a class="reference internal" href="mindspore.dataset.TextFileDataset.html">mindspore.dataset.TextFileDataset</a></li>
<li class="toctree-l3"><a class="reference internal" href="mindspore.dataset.UDPOSDataset.html">mindspore.dataset.UDPOSDataset</a></li>
<li class="toctree-l3"><a class="reference internal" href="mindspore.dataset.WikiTextDataset.html">mindspore.dataset.WikiTextDataset</a></li>
<li class="toctree-l3"><a class="reference internal" href="mindspore.dataset.YahooAnswersDataset.html">mindspore.dataset.YahooAnswersDataset</a></li>
<li class="toctree-l3"><a class="reference internal" href="mindspore.dataset.YelpReviewDataset.html">mindspore.dataset.YelpReviewDataset</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../mindspore.dataset.html#id3">音频</a></li>
<li class="toctree-l2"><a class="reference internal" href="../mindspore.dataset.html#id4">标准格式</a></li>
<li class="toctree-l2"><a class="reference internal" href="../mindspore.dataset.html#id5">用户自定义</a></li>
<li class="toctree-l2"><a class="reference internal" href="../mindspore.dataset.html#id6">图</a></li>
<li class="toctree-l2"><a class="reference internal" href="../mindspore.dataset.html#id7">采样器</a></li>
<li class="toctree-l2"><a class="reference internal" href="../mindspore.dataset.html#id8">其他</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../mindspore.dataset.audio.html">mindspore.dataset.audio</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mindspore.dataset.config.html">mindspore.dataset.config</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mindspore.dataset.text.html">mindspore.dataset.text</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mindspore.dataset.transforms.html">mindspore.dataset.transforms</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mindspore.dataset.vision.html">mindspore.dataset.vision</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mindspore.mindrecord.html">mindspore.mindrecord</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mindspore.nn.html">mindspore.nn</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mindspore.nn.probability.html">mindspore.nn.probability</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mindspore.nn.transformer.html">mindspore.nn.transformer</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mindspore.numpy.html">mindspore.numpy</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mindspore.ops.html">mindspore.ops</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mindspore.parallel.html">mindspore.parallel</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mindspore.parallel.nn.html">mindspore.parallel.nn</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mindspore.profiler.html">mindspore.profiler</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mindspore.scipy.html">mindspore.scipy</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mindspore.train.html">mindspore.train</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mindspore.boost.html">mindspore.boost</a></li>
</ul>
<p class="caption"><span class="caption-text">MindSpore C++ API</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://www.mindspore.cn/lite/api/zh-CN/master/api_cpp/mindspore.html">MindSpore Lite↗</a></li>
</ul>

            
          
        </div>
        
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" aria-label="top navigation">
        
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="../../index.html">MindSpore</a>
        
      </nav>


      <div class="wy-nav-content">
        
        <div class="rst-content">
        
          

















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../../index.html" class="icon icon-home"></a> &raquo;</li>
        
          <li><a href="../mindspore.dataset.html">mindspore.dataset</a> &raquo;</li>
        
      <li>mindspore.dataset.AGNewsDataset</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
          
            <a href="../../_sources/api_python/dataset/mindspore.dataset.AGNewsDataset.rst.txt" rel="nofollow"> View page source</a>
          
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="mindspore-dataset-agnewsdataset">
<h1>mindspore.dataset.AGNewsDataset<a class="headerlink" href="#mindspore-dataset-agnewsdataset" title="Permalink to this headline">¶</a></h1>
<dl class="class">
<dt id="mindspore.dataset.AGNewsDataset">
<em class="property">class </em><code class="sig-prename descclassname">mindspore.dataset.</code><code class="sig-name descname">AGNewsDataset</code><span class="sig-paren">(</span><em class="sig-param">dataset_dir</em>, <em class="sig-param">usage=None</em>, <em class="sig-param">num_samples=None</em>, <em class="sig-param">num_parallel_workers=None</em>, <em class="sig-param">shuffle=Shuffle.GLOBAL</em>, <em class="sig-param">num_shards=None</em>, <em class="sig-param">shard_id=None</em>, <em class="sig-param">cache=None</em><span class="sig-paren">)</span><a class="reference internal" href="../../_modules/mindspore/dataset/engine/datasets_text.html#AGNewsDataset"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#mindspore.dataset.AGNewsDataset" title="Permalink to this definition">¶</a></dt>
<dd><p>A source dataset that reads and parses AG News datasets.</p>
<p>The generated dataset has three columns: <code class="xref py py-obj docutils literal notranslate"><span class="pre">[index,</span> <span class="pre">title,</span> <span class="pre">description]</span></code>.
The tensor of column <code class="xref py py-obj docutils literal notranslate"><span class="pre">index</span></code> is of the string type.
The tensor of column <code class="xref py py-obj docutils literal notranslate"><span class="pre">title</span></code> is of the string type.
The tensor of column <code class="xref py py-obj docutils literal notranslate"><span class="pre">description</span></code> is of the string type.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>dataset_dir</strong> (<a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a>) – Path to the root directory that contains the dataset.</p></li>
<li><p><strong>usage</strong> (<a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>, </em><em>optional</em>) – Acceptable usages include <cite>train</cite>, <cite>test</cite> and <cite>all</cite> (default=None, all samples).</p></li>
<li><p><strong>num_samples</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a><em>, </em><em>optional</em>) – Number of samples (rows) to read (default=None, reads the full dataset).</p></li>
<li><p><strong>num_parallel_workers</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a><em>, </em><em>optional</em>) – Number of workers to read the data
(default=None, number set in the config).</p></li>
<li><p><strong>shuffle</strong> (<em>Union</em><em>[</em><a class="reference external" href="https://docs.python.org/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>Shuffle level</em><em>]</em><em>, </em><em>optional</em>) – <p>Perform reshuffling of the data every epoch
(default=Shuffle.GLOBAL).
If shuffle is False, no shuffling will be performed.
If shuffle is True, performs global shuffle.
There are three levels of shuffling, desired shuffle enum defined by mindspore.dataset.Shuffle.</p>
<ul>
<li><p>Shuffle.GLOBAL: Shuffle both the files and samples, same as setting shuffle to True.</p></li>
<li><p>Shuffle.FILES: Shuffle files only.</p></li>
</ul>
</p></li>
<li><p><strong>num_shards</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a><em>, </em><em>optional</em>) – Number of shards that the dataset will be divided into (default=None).
When this argument is specified, ‘num_samples’ reflects the max sample number of per shard.</p></li>
<li><p><strong>shard_id</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a><em>, </em><em>optional</em>) – The shard ID within num_shards (default=None). This
argument can only be specified when num_shards is also specified.</p></li>
<li><p><strong>cache</strong> (<a class="reference internal" href="mindspore.dataset.DatasetCache.html#mindspore.dataset.DatasetCache" title="mindspore.dataset.DatasetCache"><em>DatasetCache</em></a><em>, </em><em>optional</em>) – Use tensor caching service to speed up dataset processing.
(default=None, which means no cache is used).</p></li>
</ul>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">ag_news_dataset_dir</span> <span class="o">=</span> <span class="s2">&quot;/path/to/ag_news_dataset_file&quot;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">ds</span><span class="o">.</span><span class="n">AGNewsDataset</span><span class="p">(</span><span class="n">dataset_dir</span><span class="o">=</span><span class="n">ag_news_dataset_dir</span><span class="p">,</span> <span class="n">usage</span><span class="o">=</span><span class="s1">&#39;all&#39;</span><span class="p">)</span>
</pre></div>
</div>
<p>About AGNews dataset:</p>
<p>AG is a collection of over 1 million news articles. The news articles were collected
by ComeToMyHead from over 2,000 news sources in over 1 year of activity. ComeToMyHead
is an academic news search engine that has been in operation since July 2004.
The dataset is provided by academics for research purposes such as data mining
(clustering, classification, etc.), information retrieval (ranking, searching, etc.),
xml, data compression, data streaming, and any other non-commercial activities.
AG’s news topic classification dataset was constructed by selecting the four largest
classes from the original corpus. Each class contains 30,000 training samples and
1,900 test samples. The total number of training samples in train.csv is 120,000
and the number of test samples in test.csv is 7,600.</p>
<p>You can unzip the dataset files into the following structure and read by MindSpore’s API:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>.
└── ag_news_dataset_dir
    ├── classes.txt
    ├── train.csv
    ├── test.csv
    └── readme.txt
</pre></div>
</div>
<p>Citation:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="nd">@misc</span><span class="p">{</span><span class="n">zhang2015characterlevel</span><span class="p">,</span>
<span class="n">title</span><span class="o">=</span><span class="p">{</span><span class="n">Character</span><span class="o">-</span><span class="n">level</span> <span class="n">Convolutional</span> <span class="n">Networks</span> <span class="k">for</span> <span class="n">Text</span> <span class="n">Classification</span><span class="p">},</span>
<span class="n">author</span><span class="o">=</span><span class="p">{</span><span class="n">Xiang</span> <span class="n">Zhang</span> <span class="ow">and</span> <span class="n">Junbo</span> <span class="n">Zhao</span> <span class="ow">and</span> <span class="n">Yann</span> <span class="n">LeCun</span><span class="p">},</span>
<span class="n">year</span><span class="o">=</span><span class="p">{</span><span class="mi">2015</span><span class="p">},</span>
<span class="n">eprint</span><span class="o">=</span><span class="p">{</span><span class="mf">1509.01626</span><span class="p">},</span>
<span class="n">archivePrefix</span><span class="o">=</span><span class="p">{</span><span class="n">arXiv</span><span class="p">},</span>
<span class="n">primaryClass</span><span class="o">=</span><span class="p">{</span><span class="n">cs</span><span class="o">.</span><span class="n">LG</span><span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.apply">
<code class="sig-name descname">apply</code><span class="sig-paren">(</span><em class="sig-param">apply_func</em><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.apply" title="Permalink to this definition">¶</a></dt>
<dd><p>Apply a function in this dataset.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>apply_func</strong> (<em>function</em>) – A function that must take one ‘Dataset’ as an argument and
return a preprocessed ‘Dataset’.</p>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>Dataset, dataset applied by the function.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># dataset is an instance object of Dataset</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Declare an apply_func function which returns a Dataset object</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">apply_func</span><span class="p">(</span><span class="n">data</span><span class="p">):</span>
<span class="gp">... </span>    <span class="n">data</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">batch</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">... </span>    <span class="k">return</span> <span class="n">data</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Use apply to call apply_func</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="n">apply_func</span><span class="p">)</span>
</pre></div>
</div>
<dl class="field-list simple">
<dt class="field-odd">Raises</dt>
<dd class="field-odd"><ul class="simple">
<li><p><a class="reference external" href="https://docs.python.org/library/exceptions.html#TypeError" title="(in Python v3.8)"><strong>TypeError</strong></a> – If apply_func is not a function.</p></li>
<li><p><a class="reference external" href="https://docs.python.org/library/exceptions.html#TypeError" title="(in Python v3.8)"><strong>TypeError</strong></a> – If apply_func doesn’t return a Dataset.</p></li>
</ul>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.batch">
<code class="sig-name descname">batch</code><span class="sig-paren">(</span><em class="sig-param">batch_size</em>, <em class="sig-param">drop_remainder=False</em>, <em class="sig-param">num_parallel_workers=None</em>, <em class="sig-param">per_batch_map=None</em>, <em class="sig-param">input_columns=None</em>, <em class="sig-param">output_columns=None</em>, <em class="sig-param">column_order=None</em>, <em class="sig-param">pad_info=None</em>, <em class="sig-param">python_multiprocessing=False</em>, <em class="sig-param">max_rowsize=16</em><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.batch" title="Permalink to this definition">¶</a></dt>
<dd><p>Combine batch_size number of consecutive rows into batches.</p>
<p>For any child node, a batch is treated as a single row.
For any column, all the elements within that column must have the same shape.
If a per_batch_map callable is provided, it will be applied to the batches of tensors.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>The order of using repeat and batch reflects the number of batches and per_batch_map.
It is recommended that the repeat operation applied after the batch operation finished.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>batch_size</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a><em> or </em><em>function</em>) – The number of rows each batch is created with. An
int or callable object which takes exactly 1 parameter, BatchInfo.</p></li>
<li><p><strong>drop_remainder</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – Determines whether or not to drop the last block
whose data row number is less than batch size (default=False). If True, and if there are less
than batch_size rows available to make the last batch, then those rows will
be dropped and not propagated to the child node.</p></li>
<li><p><strong>num_parallel_workers</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a><em>, </em><em>optional</em>) – Number of workers(threads) to process the dataset in parallel
(default=None).</p></li>
<li><p><strong>per_batch_map</strong> (<em>callable</em><em>, </em><em>optional</em>) – Per batch map callable (default=None). A callable which takes
(list[Tensor], list[Tensor], …, BatchInfo) as input parameters. Each list[Tensor] represents a batch
of Tensors on a given column. The number of lists should match with the number of entries in
input_columns. The last parameter of the callable should always be a BatchInfo object. Per_batch_map
should return (list[Tensor], list[Tensor], …). The length of each list in output should be the same as
the input. output_columns is required if the number of output lists is different from input.</p></li>
<li><p><strong>input_columns</strong> (<em>Union</em><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>, </em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#list" title="(in Python v3.8)"><em>list</em></a><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>]</em><em>]</em><em>, </em><em>optional</em>) – List of names of the input columns. The size of the list
should match with signature of per_batch_map callable (default=None).</p></li>
<li><p><strong>output_columns</strong> (<em>Union</em><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>, </em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#list" title="(in Python v3.8)"><em>list</em></a><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>]</em><em>]</em><em>, </em><em>optional</em>) – List of names assigned to the columns
outputted by the last operation. This parameter is mandatory if len(input_columns) !=
len(output_columns). The size of this list must match the number of output
columns of the last operation. (default=None, output columns will have the same
name as the input columns, i.e., the columns will be replaced).</p></li>
<li><p><strong>column_order</strong> (<em>Union</em><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>, </em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#list" title="(in Python v3.8)"><em>list</em></a><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>]</em><em>]</em><em>, </em><em>optional</em>) – Specifies the list of all the columns you need in the whole
dataset (default=None). The parameter is required when len(input_column) != len(output_column).
Caution: the list here is not just the columns specified in parameter input_columns and output_columns.</p></li>
<li><p><strong>pad_info</strong> (<a class="reference external" href="https://docs.python.org/library/stdtypes.html#dict" title="(in Python v3.8)"><em>dict</em></a><em>, </em><em>optional</em>) – Whether to perform padding on selected columns. pad_info={“col1”:([224,224],0)}
would pad column with name “col1” to a tensor of size [224,224] and fill the missing with 0
(default=None).</p></li>
<li><p><strong>python_multiprocessing</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – Parallelize Python function per_batch_map with multi-processing.
This option could be beneficial if the function is computational heavy (default=False).</p></li>
<li><p><strong>max_rowsize</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a><em>, </em><em>optional</em>) – Maximum size of row in MB that is used for shared memory allocation to copy
data between processes.  This is only used if python_multiprocessing is set to True (default=16).</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>BatchDataset, dataset batched.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># Create a dataset where every 100 rows are combined into a batch</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># and drops the last incomplete batch if there is one.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">batch</span><span class="p">(</span><span class="mi">100</span><span class="p">,</span> <span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># resize image according to its batch number, if it&#39;s 5-th batch, resize to (5^2, 5^2) = (25, 25)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">np_resize</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="n">batchInfo</span><span class="p">):</span>
<span class="gp">... </span>    <span class="n">output</span> <span class="o">=</span> <span class="n">col</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
<span class="gp">... </span>    <span class="n">s</span> <span class="o">=</span> <span class="p">(</span><span class="n">batchInfo</span><span class="o">.</span><span class="n">get_batch_num</span><span class="p">()</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span>
<span class="gp">... </span>    <span class="n">index</span> <span class="o">=</span> <span class="mi">0</span>
<span class="gp">... </span>    <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">col</span><span class="p">:</span>
<span class="gp">... </span>        <span class="n">img</span> <span class="o">=</span> <span class="n">Image</span><span class="o">.</span><span class="n">fromarray</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s1">&#39;uint8&#39;</span><span class="p">))</span><span class="o">.</span><span class="n">convert</span><span class="p">(</span><span class="s1">&#39;RGB&#39;</span><span class="p">)</span>
<span class="gp">... </span>        <span class="n">img</span> <span class="o">=</span> <span class="n">img</span><span class="o">.</span><span class="n">resize</span><span class="p">((</span><span class="n">s</span><span class="p">,</span> <span class="n">s</span><span class="p">),</span> <span class="n">Image</span><span class="o">.</span><span class="n">ANTIALIAS</span><span class="p">)</span>
<span class="gp">... </span>        <span class="n">output</span><span class="p">[</span><span class="n">index</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">img</span><span class="p">)</span>
<span class="gp">... </span>        <span class="n">index</span> <span class="o">+=</span> <span class="mi">1</span>
<span class="gp">... </span>    <span class="k">return</span> <span class="p">(</span><span class="n">output</span><span class="p">,)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">batch</span><span class="p">(</span><span class="n">batch_size</span><span class="o">=</span><span class="mi">8</span><span class="p">,</span> <span class="n">input_columns</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;image&quot;</span><span class="p">],</span> <span class="n">per_batch_map</span><span class="o">=</span><span class="n">np_resize</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.bucket_batch_by_length">
<code class="sig-name descname">bucket_batch_by_length</code><span class="sig-paren">(</span><em class="sig-param">column_names</em>, <em class="sig-param">bucket_boundaries</em>, <em class="sig-param">bucket_batch_sizes</em>, <em class="sig-param">element_length_function=None</em>, <em class="sig-param">pad_info=None</em>, <em class="sig-param">pad_to_bucket_boundary=False</em>, <em class="sig-param">drop_remainder=False</em><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.bucket_batch_by_length" title="Permalink to this definition">¶</a></dt>
<dd><p>Bucket elements according to their lengths. Each bucket will be padded and batched when
they are full.</p>
<p>A length function is called on each row in the dataset. The row is then
bucketed based on its length and bucket boundaries. When a bucket reaches its
corresponding size specified in bucket_batch_sizes, the entire bucket will be
padded according to pad_info, and then form a batch.
Each batch will be full, except one special case: the last batch for each bucket may not be full.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>column_names</strong> (<a class="reference external" href="https://docs.python.org/library/stdtypes.html#list" title="(in Python v3.8)"><em>list</em></a><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>]</em>) – Columns passed to element_length_function.</p></li>
<li><p><strong>bucket_boundaries</strong> (<a class="reference external" href="https://docs.python.org/library/stdtypes.html#list" title="(in Python v3.8)"><em>list</em></a><em>[</em><a class="reference external" href="https://docs.python.org/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a><em>]</em>) – A list consisting of the upper boundaries
of the buckets. Must be strictly increasing. If there are n boundaries,
n+1 buckets are created: One bucket for [0, bucket_boundaries[0]), one
bucket for [bucket_boundaries[i], bucket_boundaries[i+1]) for each
0&lt;i&lt;n-1, and the last bucket for [bucket_boundaries[n-1], inf).</p></li>
<li><p><strong>bucket_batch_sizes</strong> (<a class="reference external" href="https://docs.python.org/library/stdtypes.html#list" title="(in Python v3.8)"><em>list</em></a><em>[</em><a class="reference external" href="https://docs.python.org/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a><em>]</em>) – A list consisting of the batch sizes for
each bucket. Must contain len(bucket_boundaries)+1 elements.</p></li>
<li><p><strong>element_length_function</strong> (<em>Callable</em><em>, </em><em>optional</em>) – A function that takes in
M arguments where M = len(column_names) and returns an integer. If no value
provided, parameter M the len(column_names) must be 1, and the size of the first
dimension of that column will be taken as the length (default=None).</p></li>
<li><p><strong>pad_info</strong> (<a class="reference external" href="https://docs.python.org/library/stdtypes.html#dict" title="(in Python v3.8)"><em>dict</em></a><em>, </em><em>optional</em>) – The information about how to batch each column. The key
corresponds to the column name, and the value must be a tuple of 2 elements.
The first element corresponds to the shape to pad to, and the second
element corresponds to the value to pad with. If a column is not
specified, then that column will be padded to the longest in the current
batch, and 0 will be used as the padding value. Any None dimensions will
be padded to the longest in the current batch, unless if
pad_to_bucket_boundary is True. If no padding is wanted, set pad_info
to None (default=None).</p></li>
<li><p><strong>pad_to_bucket_boundary</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – If True, will pad each None
dimension in pad_info to the bucket_boundary minus 1. If there are any
elements that fall into the last bucket, an error will occur
(default=False).</p></li>
<li><p><strong>drop_remainder</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – If True, will drop the last batch for each
bucket if it is not a full batch (default=False).</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>Dataset, dataset bucketed and batched by length.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># Create a dataset where certain counts rows are combined into a batch</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># and drops the last incomplete batch if there is one.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">generate_2_columns</span><span class="p">(</span><span class="n">n</span><span class="p">):</span>
<span class="gp">... </span>    <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">n</span><span class="p">):</span>
<span class="gp">... </span>        <span class="k">yield</span> <span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">i</span><span class="p">]),</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">j</span> <span class="k">for</span> <span class="n">j</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)]))</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">column_names</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;col1&quot;</span><span class="p">,</span> <span class="s2">&quot;col2&quot;</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">ds</span><span class="o">.</span><span class="n">GeneratorDataset</span><span class="p">(</span><span class="n">generate_2_columns</span><span class="p">(</span><span class="mi">8</span><span class="p">),</span> <span class="n">column_names</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">bucket_boundaries</span> <span class="o">=</span> <span class="p">[</span><span class="mi">5</span><span class="p">,</span> <span class="mi">10</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">bucket_batch_sizes</span> <span class="o">=</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">element_length_function</span> <span class="o">=</span> <span class="p">(</span><span class="k">lambda</span> <span class="n">col1</span><span class="p">,</span> <span class="n">col2</span><span class="p">:</span> <span class="nb">max</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">col1</span><span class="p">),</span> <span class="nb">len</span><span class="p">(</span><span class="n">col2</span><span class="p">)))</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Will pad col2 to shape [bucket_boundaries[i]] where i is the</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># index of the bucket that is currently being batched.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">pad_info</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;col2&quot;</span><span class="p">:</span> <span class="p">([</span><span class="kc">None</span><span class="p">],</span> <span class="o">-</span><span class="mi">1</span><span class="p">)}</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">pad_to_bucket_boundary</span> <span class="o">=</span> <span class="kc">True</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">bucket_batch_by_length</span><span class="p">(</span><span class="n">column_names</span><span class="p">,</span> <span class="n">bucket_boundaries</span><span class="p">,</span>
<span class="gp">... </span>                                         <span class="n">bucket_batch_sizes</span><span class="p">,</span>
<span class="gp">... </span>                                         <span class="n">element_length_function</span><span class="p">,</span> <span class="n">pad_info</span><span class="p">,</span>
<span class="gp">... </span>                                         <span class="n">pad_to_bucket_boundary</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.build_sentencepiece_vocab">
<code class="sig-name descname">build_sentencepiece_vocab</code><span class="sig-paren">(</span><em class="sig-param">columns</em>, <em class="sig-param">vocab_size</em>, <em class="sig-param">character_coverage</em>, <em class="sig-param">model_type</em>, <em class="sig-param">params</em><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.build_sentencepiece_vocab" title="Permalink to this definition">¶</a></dt>
<dd><p>Function to create a SentencePieceVocab from source dataset.
Desired source dataset is a text type dataset.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>columns</strong> (<a class="reference external" href="https://docs.python.org/library/stdtypes.html#list" title="(in Python v3.8)"><em>list</em></a><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>]</em>) – Column names to get words from.</p></li>
<li><p><strong>vocab_size</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a>) – Vocabulary size.</p></li>
<li><p><strong>character_coverage</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a>) – Percentage of characters covered by the model, must be between
0.98 and 1.0 Good defaults are: 0.9995 for languages with rich character sets like
Japanese or Chinese character sets, and 1.0 for other languages with small character sets
like English or Latin.</p></li>
<li><p><strong>model_type</strong> (<a class="reference internal" href="../dataset_text/mindspore.dataset.text.SentencePieceModel.html#mindspore.dataset.text.SentencePieceModel" title="mindspore.dataset.text.SentencePieceModel"><em>SentencePieceModel</em></a>) – Model type. Choose from unigram (default), bpe, char, or word.
The input sentence must be pretokenized when using word type.</p></li>
<li><p><strong>params</strong> (<a class="reference external" href="https://docs.python.org/library/stdtypes.html#dict" title="(in Python v3.8)"><em>dict</em></a>) – Any extra optional parameters of sentencepiece library according to your raw data</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>SentencePieceVocab, vocab built from the dataset.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">mindspore.dataset.text</span> <span class="kn">import</span> <span class="n">SentencePieceModel</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># You can construct any text dataset as source, take TextFileDataset as example.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">ds</span><span class="o">.</span><span class="n">TextFileDataset</span><span class="p">(</span><span class="s2">&quot;/path/to/sentence/piece/vocab/file&quot;</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">build_sentencepiece_vocab</span><span class="p">([</span><span class="s2">&quot;text&quot;</span><span class="p">],</span> <span class="mi">5000</span><span class="p">,</span> <span class="mf">0.9995</span><span class="p">,</span> <span class="n">SentencePieceModel</span><span class="o">.</span><span class="n">UNIGRAM</span><span class="p">,</span> <span class="p">{})</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.build_vocab">
<code class="sig-name descname">build_vocab</code><span class="sig-paren">(</span><em class="sig-param">columns</em>, <em class="sig-param">freq_range</em>, <em class="sig-param">top_k</em>, <em class="sig-param">special_tokens</em>, <em class="sig-param">special_first</em><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.build_vocab" title="Permalink to this definition">¶</a></dt>
<dd><p>Function to create a Vocab from source dataset.
Desired source dataset is a text type dataset.</p>
<p>Build a vocab from a dataset. This would collect all the unique words in a dataset and return a vocab
which contains top_k most frequent words (if top_k is specified)</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>columns</strong> (<em>Union</em><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>, </em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#list" title="(in Python v3.8)"><em>list</em></a><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>]</em><em>]</em>) – Column names to get words from.</p></li>
<li><p><strong>freq_range</strong> (<a class="reference external" href="https://docs.python.org/library/stdtypes.html#tuple" title="(in Python v3.8)"><em>tuple</em></a><em>[</em><a class="reference external" href="https://docs.python.org/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a><em>]</em>) – A tuple of integers (min_frequency, max_frequency). Words within the frequency
range will be stored.
Naturally 0 &lt;= min_frequency &lt;= max_frequency &lt;= total_words. min_frequency/max_frequency
can be set to default, which corresponds to 0/total_words separately.</p></li>
<li><p><strong>top_k</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a>) – Number of words to be built into vocab. top_k most frequent words are
taken. The top_k is taken after freq_range. If not enough top_k, all words will be taken</p></li>
<li><p><strong>special_tokens</strong> (<a class="reference external" href="https://docs.python.org/library/stdtypes.html#list" title="(in Python v3.8)"><em>list</em></a><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>]</em>) – A list of strings, each one is a special token.</p></li>
<li><p><strong>special_first</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a>) – Whether special_tokens will be prepended/appended to vocab, If special_tokens
is specified and special_first is set to default, special_tokens will be prepended.</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>Vocab, vocab built from the dataset.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">gen_corpus</span><span class="p">():</span>
<span class="gp">... </span>    <span class="c1"># key: word, value: number of occurrences, reason for using letters is so their order is apparent</span>
<span class="gp">... </span>    <span class="n">corpus</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;Z&quot;</span><span class="p">:</span> <span class="mi">4</span><span class="p">,</span> <span class="s2">&quot;Y&quot;</span><span class="p">:</span> <span class="mi">4</span><span class="p">,</span> <span class="s2">&quot;X&quot;</span><span class="p">:</span> <span class="mi">4</span><span class="p">,</span> <span class="s2">&quot;W&quot;</span><span class="p">:</span> <span class="mi">3</span><span class="p">,</span> <span class="s2">&quot;U&quot;</span><span class="p">:</span> <span class="mi">3</span><span class="p">,</span> <span class="s2">&quot;V&quot;</span><span class="p">:</span> <span class="mi">2</span><span class="p">,</span> <span class="s2">&quot;T&quot;</span><span class="p">:</span> <span class="mi">1</span><span class="p">}</span>
<span class="gp">... </span>    <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">corpus</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="gp">... </span>        <span class="k">yield</span> <span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">k</span><span class="p">]</span> <span class="o">*</span> <span class="n">v</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="s1">&#39;S&#39;</span><span class="p">),)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">column_names</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;column1&quot;</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">ds</span><span class="o">.</span><span class="n">GeneratorDataset</span><span class="p">(</span><span class="n">gen_corpus</span><span class="p">,</span> <span class="n">column_names</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">build_vocab</span><span class="p">(</span><span class="n">columns</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;column1&quot;</span><span class="p">],</span>
<span class="gp">... </span>                              <span class="n">freq_range</span><span class="o">=</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">10</span><span class="p">),</span> <span class="n">top_k</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span>
<span class="gp">... </span>                              <span class="n">special_tokens</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;&lt;pad&gt;&quot;</span><span class="p">,</span> <span class="s2">&quot;&lt;unk&gt;&quot;</span><span class="p">],</span>
<span class="gp">... </span>                              <span class="n">special_first</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.close_pool">
<code class="sig-name descname">close_pool</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.close_pool" title="Permalink to this definition">¶</a></dt>
<dd><p>Close multiprocessing pool in dataset. If you are familiar with multiprocessing library, you can regard this
as a destructor for a processingPool object.</p>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.concat">
<code class="sig-name descname">concat</code><span class="sig-paren">(</span><em class="sig-param">datasets</em><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.concat" title="Permalink to this definition">¶</a></dt>
<dd><p>Concatenate the dataset objects in the input list.
Performing “+” operation on dataset objects can achieve the same effect.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>The column name, and rank and type of the column data must be the same in the input datasets.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>datasets</strong> (<em>Union</em><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#list" title="(in Python v3.8)"><em>list</em></a><em>, </em><em>class Dataset</em><em>]</em>) – A list of datasets or a single class Dataset
to be concatenated together with this dataset.</p>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>Dataset, dataset concatenated.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># Create a dataset by concatenating dataset_1 and dataset_2 with &quot;+&quot; operator</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset_1</span> <span class="o">+</span> <span class="n">dataset_2</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Create a dataset by concatenating dataset_1 and dataset_2 with concat operation</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset_1</span><span class="o">.</span><span class="n">concat</span><span class="p">(</span><span class="n">dataset_2</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.create_dict_iterator">
<code class="sig-name descname">create_dict_iterator</code><span class="sig-paren">(</span><em class="sig-param">num_epochs=-1</em>, <em class="sig-param">output_numpy=False</em><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.create_dict_iterator" title="Permalink to this definition">¶</a></dt>
<dd><p>Create an iterator over the dataset. The data retrieved will be a dictionary datatype.</p>
<p>The order of the columns in the dictionary may not be the same as the original order.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>num_epochs</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a><em>, </em><em>optional</em>) – Maximum number of epochs that iterator can be iterated
(default=-1, iterator can be iterated infinite number of epochs).</p></li>
<li><p><strong>output_numpy</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – Whether or not to output NumPy datatype,
if output_numpy=False, iterator will output MSTensor (default=False).</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>Iterator, dictionary iterator over the dataset.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># dataset is an instance object of Dataset</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">iterator</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">create_dict_iterator</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">iterator</span><span class="p">:</span>
<span class="gp">... </span>    <span class="c1"># item is a dict</span>
<span class="gp">... </span>    <span class="nb">print</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">item</span><span class="p">))</span>
<span class="gp">... </span>    <span class="k">break</span>
<span class="go">&lt;class &#39;dict&#39;&gt;</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.create_tuple_iterator">
<code class="sig-name descname">create_tuple_iterator</code><span class="sig-paren">(</span><em class="sig-param">columns=None</em>, <em class="sig-param">num_epochs=-1</em>, <em class="sig-param">output_numpy=False</em>, <em class="sig-param">do_copy=True</em><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.create_tuple_iterator" title="Permalink to this definition">¶</a></dt>
<dd><p>Create an iterator over the dataset. The datatype retrieved back will be a list of ndarrays.</p>
<p>To specify which columns to list and the order needed, use columns_list. If columns_list
is not provided, the order of the columns will remain unchanged.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>columns</strong> (<a class="reference external" href="https://docs.python.org/library/stdtypes.html#list" title="(in Python v3.8)"><em>list</em></a><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>]</em><em>, </em><em>optional</em>) – List of columns to be used to specify the order of columns
(default=None, means all columns).</p></li>
<li><p><strong>num_epochs</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a><em>, </em><em>optional</em>) – Maximum number of epochs that iterator can be iterated.
(default=-1, iterator can be iterated infinite number of epochs)</p></li>
<li><p><strong>output_numpy</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – Whether or not to output NumPy datatype.
If output_numpy=False, iterator will output MSTensor (default=False).</p></li>
<li><p><strong>do_copy</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – when output data type is mindspore.Tensor,
use this param to select the conversion method, only take False for better performance (default=True).</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>Iterator, tuple iterator over the dataset.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># dataset is an instance object of Dataset</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">iterator</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">create_tuple_iterator</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">iterator</span><span class="p">:</span>
<span class="gp">... </span>    <span class="c1"># item is a list</span>
<span class="gp">... </span>    <span class="nb">print</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">item</span><span class="p">))</span>
<span class="gp">... </span>    <span class="k">break</span>
<span class="go">&lt;class &#39;list&#39;&gt;</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.device_que">
<code class="sig-name descname">device_que</code><span class="sig-paren">(</span><em class="sig-param">send_epoch_end=True</em>, <em class="sig-param">create_data_info_queue=False</em><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.device_que" title="Permalink to this definition">¶</a></dt>
<dd><p>Return a transferred Dataset that transfers data through a device.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>send_epoch_end</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – Whether to send end of sequence to device or not (default=True).</p></li>
<li><p><strong>create_data_info_queue</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – Whether to create queue which stores
types and shapes of data or not(default=False).</p></li>
</ul>
</dd>
</dl>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>If device is Ascend, features of data will be transferred one by one. The limitation
of data transmission per time is 256M.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>Dataset, dataset for transferring.</p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.dynamic_min_max_shapes">
<code class="sig-name descname">dynamic_min_max_shapes</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.dynamic_min_max_shapes" title="Permalink to this definition">¶</a></dt>
<dd><p>Get minimum and maximum data length of dynamic source data, for dynamic graph compilation.</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>lists, min_shapes, max_shapes of source data.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">generator1</span><span class="p">():</span>
<span class="gp">... </span>    <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">100</span><span class="p">):</span>
<span class="gp">... </span>        <span class="k">yield</span> <span class="n">np</span><span class="o">.</span><span class="n">ones</span><span class="p">((</span><span class="mi">16</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="mi">83</span><span class="p">)),</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">i</span><span class="p">)</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">ds</span><span class="o">.</span><span class="n">GeneratorDataset</span><span class="p">(</span><span class="n">generator1</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;data1&quot;</span><span class="p">,</span> <span class="s2">&quot;data2&quot;</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span><span class="o">.</span><span class="n">set_dynamic_columns</span><span class="p">(</span><span class="n">columns</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;data1&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mi">16</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="mi">83</span><span class="p">],</span> <span class="s2">&quot;data2&quot;</span><span class="p">:</span> <span class="p">[]})</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">min_shapes</span><span class="p">,</span> <span class="n">max_shapes</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">dynamic_min_max_shapes</span><span class="p">()</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.filter">
<code class="sig-name descname">filter</code><span class="sig-paren">(</span><em class="sig-param">predicate</em>, <em class="sig-param">input_columns=None</em>, <em class="sig-param">num_parallel_workers=None</em><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.filter" title="Permalink to this definition">¶</a></dt>
<dd><p>Filter dataset by prediction.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>If input_columns not provided or provided with empty, all columns will be used.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>predicate</strong> (<em>callable</em>) – Python callable which returns a boolean value. If False then filter the element.</p></li>
<li><p><strong>input_columns</strong> (<em>Union</em><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>, </em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#list" title="(in Python v3.8)"><em>list</em></a><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>]</em><em>]</em><em>, </em><em>optional</em>) – List of names of the input columns, when
default=None, the predicate will be applied on all columns in the dataset.</p></li>
<li><p><strong>num_parallel_workers</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a><em>, </em><em>optional</em>) – Number of workers to process the dataset
in parallel (default=None).</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>Dataset, dataset filtered.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># generator data(0 ~ 63)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># filter the data that greater than or equal to 11</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">predicate</span><span class="o">=</span><span class="k">lambda</span> <span class="n">data</span><span class="p">:</span> <span class="n">data</span> <span class="o">&lt;</span> <span class="mi">11</span><span class="p">,</span> <span class="n">input_columns</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;data&quot;</span><span class="p">])</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.flat_map">
<code class="sig-name descname">flat_map</code><span class="sig-paren">(</span><em class="sig-param">func</em><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.flat_map" title="Permalink to this definition">¶</a></dt>
<dd><p>Map <cite>func</cite> to each row in dataset and flatten the result.</p>
<p>The specified <cite>func</cite> is a function that must take one ‘Ndarray’ as input
and return a ‘Dataset’.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>func</strong> (<em>function</em>) – A function that must take one ‘Ndarray’ as an argument and
return a ‘Dataset’.</p>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>Dataset, dataset applied by the function.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># use NumpySlicesDataset as an example</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">ds</span><span class="o">.</span><span class="n">NumpySlicesDataset</span><span class="p">([[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]])</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">flat_map_func</span><span class="p">(</span><span class="n">array</span><span class="p">):</span>
<span class="gp">... </span>    <span class="c1"># create a NumpySlicesDataset with the array</span>
<span class="gp">... </span>    <span class="n">dataset</span> <span class="o">=</span> <span class="n">ds</span><span class="o">.</span><span class="n">NumpySlicesDataset</span><span class="p">(</span><span class="n">array</span><span class="p">)</span>
<span class="gp">... </span>    <span class="c1"># repeat the dataset twice</span>
<span class="gp">... </span>    <span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">repeat</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">... </span>    <span class="k">return</span> <span class="n">dataset</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">flat_map</span><span class="p">(</span><span class="n">flat_map_func</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># [[0, 1], [0, 1], [2, 3], [2, 3]]</span>
</pre></div>
</div>
<dl class="field-list simple">
<dt class="field-odd">Raises</dt>
<dd class="field-odd"><ul class="simple">
<li><p><a class="reference external" href="https://docs.python.org/library/exceptions.html#TypeError" title="(in Python v3.8)"><strong>TypeError</strong></a> – If <cite>func</cite> is not a function.</p></li>
<li><p><a class="reference external" href="https://docs.python.org/library/exceptions.html#TypeError" title="(in Python v3.8)"><strong>TypeError</strong></a> – If <cite>func</cite> doesn’t return a Dataset.</p></li>
</ul>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.get_batch_size">
<code class="sig-name descname">get_batch_size</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.get_batch_size" title="Permalink to this definition">¶</a></dt>
<dd><p>Return the size of batch.</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>int, the number of data in a batch.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># dataset is an instance object of Dataset</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">batch_size</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">get_batch_size</span><span class="p">()</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.get_class_indexing">
<code class="sig-name descname">get_class_indexing</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.get_class_indexing" title="Permalink to this definition">¶</a></dt>
<dd><p>Return the class index.</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>dict, a str-to-int mapping from label name to index.
dict, a str-to-list&lt;int&gt; mapping from label name to index for Coco ONLY. The second number
in the list is used to indicate the super category.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># dataset is an instance object of Dataset</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">class_indexing</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">get_class_indexing</span><span class="p">()</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.get_col_names">
<code class="sig-name descname">get_col_names</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.get_col_names" title="Permalink to this definition">¶</a></dt>
<dd><p>Return the names of the columns in dataset.</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>list, list of column names in the dataset.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># dataset is an instance object of Dataset</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">col_names</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">get_col_names</span><span class="p">()</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.get_dataset_size">
<code class="sig-name descname">get_dataset_size</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.get_dataset_size" title="Permalink to this definition">¶</a></dt>
<dd><p>Return the number of batches in an epoch.</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>int, number of batches.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># dataset is an instance object of Dataset</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset_size</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">get_dataset_size</span><span class="p">()</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.get_repeat_count">
<code class="sig-name descname">get_repeat_count</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.get_repeat_count" title="Permalink to this definition">¶</a></dt>
<dd><p>Get the replication times in RepeatDataset (default is 1).</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>int, the count of repeat.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># dataset is an instance object of Dataset</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">repeat_count</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">get_repeat_count</span><span class="p">()</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.input_indexs">
<em class="property">property </em><code class="sig-name descname">input_indexs</code><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.input_indexs" title="Permalink to this definition">¶</a></dt>
<dd><p>Get Input Index Information</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>tuple, tuple of the input index information.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># dataset is an instance object of Dataset</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># set input_indexs</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span><span class="o">.</span><span class="n">input_indexs</span> <span class="o">=</span> <span class="mi">10</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">print</span><span class="p">(</span><span class="n">dataset</span><span class="o">.</span><span class="n">input_indexs</span><span class="p">)</span>
<span class="go">10</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.map">
<code class="sig-name descname">map</code><span class="sig-paren">(</span><em class="sig-param">operations</em>, <em class="sig-param">input_columns=None</em>, <em class="sig-param">output_columns=None</em>, <em class="sig-param">column_order=None</em>, <em class="sig-param">num_parallel_workers=None</em>, <em class="sig-param">python_multiprocessing=False</em>, <em class="sig-param">cache=None</em>, <em class="sig-param">callbacks=None</em>, <em class="sig-param">max_rowsize=16</em>, <em class="sig-param">offload=None</em><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.map" title="Permalink to this definition">¶</a></dt>
<dd><p>Apply each operation in operations to this dataset.</p>
<p>The order of operations is determined by the position of each operation in the operations parameter.
operations[0] will be applied first, then operations[1], then operations[2], etc.</p>
<p>Each operation will be passed one or more columns from the dataset as input, and zero or
more columns will be outputted. The first operation will be passed the columns specified
in input_columns as input. If there is more than one operator in operations, the outputted
columns of the previous operation are used as the input columns for the next operation.
The columns outputted by the very last operation will be assigned names specified by
output_columns.</p>
<p>Only the columns specified in column_order will be propagated to the child node. These
columns will be in the same order as specified in column_order.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>operations</strong> (<em>Union</em><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#list" title="(in Python v3.8)"><em>list</em></a><em>[</em><em>TensorOp</em><em>]</em><em>, </em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#list" title="(in Python v3.8)"><em>list</em></a><em>[</em><em>functions</em><em>]</em><em>]</em>) – List of operations to be
applied on the dataset. Operations are applied in the order they appear in this list.</p></li>
<li><p><strong>input_columns</strong> (<em>Union</em><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>, </em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#list" title="(in Python v3.8)"><em>list</em></a><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>]</em><em>]</em><em>, </em><em>optional</em>) – List of the names of the columns that will be passed to
the first operation as input. The size of this list must match the number of
input columns expected by the first operator. (default=None, the first
operation will be passed however many columns that are required, starting from
the first column).</p></li>
<li><p><strong>output_columns</strong> (<em>Union</em><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>, </em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#list" title="(in Python v3.8)"><em>list</em></a><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>]</em><em>]</em><em>, </em><em>optional</em>) – List of names assigned to the columns outputted by
the last operation. This parameter is mandatory if len(input_columns) !=
len(output_columns). The size of this list must match the number of output
columns of the last operation. (default=None, output columns will have the same
name as the input columns, i.e., the columns will be replaced).</p></li>
<li><p><strong>column_order</strong> (<a class="reference external" href="https://docs.python.org/library/stdtypes.html#list" title="(in Python v3.8)"><em>list</em></a><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>]</em><em>, </em><em>optional</em>) – Specifies the list of all the columns you need in the whole
dataset (default=None). The parameter is required when len(input_column) != len(output_column).
Caution: the list here is not just the columns specified in parameter input_columns and output_columns.</p></li>
<li><p><strong>num_parallel_workers</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a><em>, </em><em>optional</em>) – Number of threads used to process the dataset in
parallel (default=None, the value from the configuration will be used).</p></li>
<li><p><strong>python_multiprocessing</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – Parallelize Python operations with multiple worker processes. This
option could be beneficial if the Python operation is computational heavy (default=False).</p></li>
<li><p><strong>cache</strong> (<a class="reference internal" href="mindspore.dataset.DatasetCache.html#mindspore.dataset.DatasetCache" title="mindspore.dataset.DatasetCache"><em>DatasetCache</em></a><em>, </em><em>optional</em>) – Use tensor caching service to speed up dataset processing.
(default=None, which means no cache is used).</p></li>
<li><p><strong>callbacks</strong> (<a class="reference internal" href="mindspore.dataset.DSCallback.html#mindspore.dataset.DSCallback" title="mindspore.dataset.DSCallback"><em>DSCallback</em></a><em>, </em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#list" title="(in Python v3.8)"><em>list</em></a><em>[</em><a class="reference internal" href="mindspore.dataset.DSCallback.html#mindspore.dataset.DSCallback" title="mindspore.dataset.DSCallback"><em>DSCallback</em></a><em>]</em><em>, </em><em>optional</em>) – List of Dataset callbacks to be called (Default=None).</p></li>
<li><p><strong>max_rowsize</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a><em>, </em><em>optional</em>) – Maximum size of row in MB that is used for shared memory allocation to copy
data between processes.  This is only used if python_multiprocessing is set to True (Default=16).</p></li>
<li><p><strong>offload</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – Flag to indicate whether offload is used (Default=None).</p></li>
</ul>
</dd>
</dl>
<div class="admonition note">
<p class="admonition-title">Note</p>
<ul class="simple">
<li><p>Input <cite>operations</cite> mainly accept c_transforms, py_transforms operator in mindspore.dataset part, plus user
defined Python function(PyFuncs).</p></li>
<li><p>Do not add network computing operators from mindspore.nn and mindspore.ops or others into this
<cite>operations</cite>.</p></li>
</ul>
</div>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>Dataset, dataset after mapping operation.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># dataset is an instance of Dataset which has 2 columns, &quot;image&quot; and &quot;label&quot;.</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Define two operations, where each operation accepts 1 input column and outputs 1 column.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">decode_op</span> <span class="o">=</span> <span class="n">c_vision</span><span class="o">.</span><span class="n">Decode</span><span class="p">(</span><span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">random_jitter_op</span> <span class="o">=</span> <span class="n">c_vision</span><span class="o">.</span><span class="n">RandomColorAdjust</span><span class="p">(</span><span class="n">brightness</span><span class="o">=</span><span class="p">(</span><span class="mf">0.8</span><span class="p">,</span> <span class="mf">0.8</span><span class="p">),</span> <span class="n">contrast</span><span class="o">=</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span>
<span class="gp">... </span>                                              <span class="n">saturation</span><span class="o">=</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="n">hue</span><span class="o">=</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">))</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># 1) Simple map example.</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Apply decode_op on column &quot;image&quot;. This column will be replaced by the outputted</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># column of decode_op. Since column_order is not provided, both columns &quot;image&quot;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># and &quot;label&quot; will be propagated to the child node in their original order.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">operations</span><span class="o">=</span><span class="p">[</span><span class="n">decode_op</span><span class="p">],</span> <span class="n">input_columns</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;image&quot;</span><span class="p">])</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Decode and rename column &quot;image&quot; to &quot;decoded_image&quot;.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">operations</span><span class="o">=</span><span class="p">[</span><span class="n">decode_op</span><span class="p">],</span> <span class="n">input_columns</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;image&quot;</span><span class="p">],</span> <span class="n">output_columns</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;decoded_image&quot;</span><span class="p">])</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Specify the order of the output columns.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">operations</span><span class="o">=</span><span class="p">[</span><span class="n">decode_op</span><span class="p">],</span> <span class="n">input_columns</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;image&quot;</span><span class="p">],</span>
<span class="gp">... </span>                      <span class="n">output_columns</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">column_order</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;label&quot;</span><span class="p">,</span> <span class="s2">&quot;image&quot;</span><span class="p">])</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Rename column &quot;image&quot; to &quot;decoded_image&quot; and also specify the order of the output columns.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">operations</span><span class="o">=</span><span class="p">[</span><span class="n">decode_op</span><span class="p">],</span> <span class="n">input_columns</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;image&quot;</span><span class="p">],</span>
<span class="gp">... </span>                      <span class="n">output_columns</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;decoded_image&quot;</span><span class="p">],</span> <span class="n">column_order</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;label&quot;</span><span class="p">,</span> <span class="s2">&quot;decoded_image&quot;</span><span class="p">])</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Rename column &quot;image&quot; to &quot;decoded_image&quot; and keep only this column.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">operations</span><span class="o">=</span><span class="p">[</span><span class="n">decode_op</span><span class="p">],</span> <span class="n">input_columns</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;image&quot;</span><span class="p">],</span>
<span class="gp">... </span>                      <span class="n">output_columns</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;decoded_image&quot;</span><span class="p">],</span> <span class="n">column_order</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;decoded_image&quot;</span><span class="p">])</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># A simple example for mapping pyfunc. Renaming columns and specifying column order</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># work in the same way as the previous examples.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">ds</span><span class="o">.</span><span class="n">NumpySlicesDataset</span><span class="p">(</span><span class="n">data</span><span class="o">=</span><span class="p">[[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">]],</span> <span class="n">column_names</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;data&quot;</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">operations</span><span class="o">=</span><span class="p">[(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)],</span> <span class="n">input_columns</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;data&quot;</span><span class="p">])</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># 2) Map example with more than one operation.</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Create a dataset where the images are decoded, then randomly color jittered.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># decode_op takes column &quot;image&quot; as input and outputs one column. The column</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># outputted by decode_op is passed as input to random_jitter_op.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># random_jitter_op will output one column. Column &quot;image&quot; will be replaced by</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># the column outputted by random_jitter_op (the very last operation). All other</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># columns are unchanged. Since column_order is not specified, the order of the</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># columns will remain the same.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">operations</span><span class="o">=</span><span class="p">[</span><span class="n">decode_op</span><span class="p">,</span> <span class="n">random_jitter_op</span><span class="p">],</span> <span class="n">input_columns</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;image&quot;</span><span class="p">])</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Rename the column outputted by random_jitter_op to &quot;image_mapped&quot;.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Specifying column order works in the same way as examples in 1).</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">operations</span><span class="o">=</span><span class="p">[</span><span class="n">decode_op</span><span class="p">,</span> <span class="n">random_jitter_op</span><span class="p">],</span> <span class="n">input_columns</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;image&quot;</span><span class="p">],</span>
<span class="gp">... </span>                      <span class="n">output_columns</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;image_mapped&quot;</span><span class="p">])</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Map with multiple operations using pyfunc. Renaming columns and specifying column order</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># work in the same way as examples in 1).</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">ds</span><span class="o">.</span><span class="n">NumpySlicesDataset</span><span class="p">(</span><span class="n">data</span><span class="o">=</span><span class="p">[[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">]],</span> <span class="n">column_names</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;data&quot;</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">operations</span><span class="o">=</span><span class="p">[(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span> <span class="o">*</span> <span class="n">x</span><span class="p">),</span> <span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)],</span> <span class="n">input_columns</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;data&quot;</span><span class="p">],</span>
<span class="gp">... </span>                                  <span class="n">output_columns</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;data_mapped&quot;</span><span class="p">])</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># 3) Example where number of input columns is not equal to number of output columns.</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># operations[0] is a lambda that takes 2 columns as input and outputs 3 columns.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># operations[1] is a lambda that takes 3 columns as input and outputs 1 column.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># operations[2] is a lambda that takes 1 column as input and outputs 4 columns.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1">#</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Note: The number of output columns of operation[i] must equal the number of</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># input columns of operation[i+1]. Otherwise, this map call will also result</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># in an error.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">operations</span> <span class="o">=</span> <span class="p">[(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">:</span> <span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">x</span> <span class="o">+</span> <span class="n">y</span><span class="p">,</span> <span class="n">x</span> <span class="o">+</span> <span class="n">y</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)),</span>
<span class="gp">... </span>              <span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">z</span><span class="p">:</span> <span class="n">x</span> <span class="o">*</span> <span class="n">y</span> <span class="o">*</span> <span class="n">z</span><span class="p">),</span>
<span class="gp">... </span>              <span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="p">(</span><span class="n">x</span> <span class="o">%</span> <span class="mi">2</span><span class="p">,</span> <span class="n">x</span> <span class="o">%</span> <span class="mi">3</span><span class="p">,</span> <span class="n">x</span> <span class="o">%</span> <span class="mi">5</span><span class="p">,</span> <span class="n">x</span> <span class="o">%</span> <span class="mi">7</span><span class="p">))]</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Note: Since the number of input columns is not the same as the number of</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># output columns, the output_columns and column_order parameters must be</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># specified. Otherwise, this map call will also result in an error.</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">ds</span><span class="o">.</span><span class="n">NumpySlicesDataset</span><span class="p">(</span><span class="n">data</span><span class="o">=</span><span class="p">([[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">]],</span> <span class="p">[[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">5</span><span class="p">]]),</span> <span class="n">column_names</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;x&quot;</span><span class="p">,</span> <span class="s2">&quot;y&quot;</span><span class="p">])</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Propagate all columns to the child node in this order:</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">operations</span><span class="p">,</span> <span class="n">input_columns</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;x&quot;</span><span class="p">,</span> <span class="s2">&quot;y&quot;</span><span class="p">],</span>
<span class="gp">... </span>                      <span class="n">output_columns</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;mod2&quot;</span><span class="p">,</span> <span class="s2">&quot;mod3&quot;</span><span class="p">,</span> <span class="s2">&quot;mod5&quot;</span><span class="p">,</span> <span class="s2">&quot;mod7&quot;</span><span class="p">],</span>
<span class="gp">... </span>                      <span class="n">column_order</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;mod2&quot;</span><span class="p">,</span> <span class="s2">&quot;mod3&quot;</span><span class="p">,</span> <span class="s2">&quot;mod5&quot;</span><span class="p">,</span> <span class="s2">&quot;mod7&quot;</span><span class="p">])</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Propagate some columns to the child node in this order:</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">operations</span><span class="p">,</span> <span class="n">input_columns</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;x&quot;</span><span class="p">,</span> <span class="s2">&quot;y&quot;</span><span class="p">],</span>
<span class="gp">... </span>                      <span class="n">output_columns</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;mod2&quot;</span><span class="p">,</span> <span class="s2">&quot;mod3&quot;</span><span class="p">,</span> <span class="s2">&quot;mod5&quot;</span><span class="p">,</span> <span class="s2">&quot;mod7&quot;</span><span class="p">],</span>
<span class="gp">... </span>                      <span class="n">column_order</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;mod7&quot;</span><span class="p">,</span> <span class="s2">&quot;mod3&quot;</span><span class="p">,</span> <span class="s2">&quot;col2&quot;</span><span class="p">])</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.notify_watchdog">
<code class="sig-name descname">notify_watchdog</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.notify_watchdog" title="Permalink to this definition">¶</a></dt>
<dd><p>Close watchdog thread in dataset. Now GeneratorDataset/map/batch will use a thread named watch_dog to monitor
multiprocess, for get_dataset_size/output_shapes/output_types/get_col_name/num_classes, we need notify_watchdog
to close watch_dog thread manually.</p>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.num_classes">
<code class="sig-name descname">num_classes</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.num_classes" title="Permalink to this definition">¶</a></dt>
<dd><p>Get the number of classes in a dataset.</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>int, number of classes.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># dataset is an instance object of Dataset</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">num_classes</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">num_classes</span><span class="p">()</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.output_shapes">
<code class="sig-name descname">output_shapes</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.output_shapes" title="Permalink to this definition">¶</a></dt>
<dd><p>Get the shapes of output data.</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>list, list of shapes of each column.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># dataset is an instance object of Dataset</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output_shapes</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">output_shapes</span><span class="p">()</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.output_types">
<code class="sig-name descname">output_types</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.output_types" title="Permalink to this definition">¶</a></dt>
<dd><p>Get the types of output data.</p>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>list, list of data types.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># dataset is an instance object of Dataset</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output_types</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">output_types</span><span class="p">()</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.project">
<code class="sig-name descname">project</code><span class="sig-paren">(</span><em class="sig-param">columns</em><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.project" title="Permalink to this definition">¶</a></dt>
<dd><p>Project certain columns in input dataset.</p>
<p>The specified columns will be selected from the dataset and passed into
the pipeline with the order specified. The other columns are discarded.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>columns</strong> (<em>Union</em><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>, </em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#list" title="(in Python v3.8)"><em>list</em></a><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>]</em><em>]</em>) – List of names of the columns to project.</p>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>Dataset, dataset projected.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># dataset is an instance object of Dataset</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">columns_to_project</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;column3&quot;</span><span class="p">,</span> <span class="s2">&quot;column1&quot;</span><span class="p">,</span> <span class="s2">&quot;column2&quot;</span><span class="p">]</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Create a dataset that consists of column3, column1, column2</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># in that order, regardless of the original order of columns.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">project</span><span class="p">(</span><span class="n">columns</span><span class="o">=</span><span class="n">columns_to_project</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.rename">
<code class="sig-name descname">rename</code><span class="sig-paren">(</span><em class="sig-param">input_columns</em>, <em class="sig-param">output_columns</em><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.rename" title="Permalink to this definition">¶</a></dt>
<dd><p>Rename the columns in input datasets.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>input_columns</strong> (<em>Union</em><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>, </em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#list" title="(in Python v3.8)"><em>list</em></a><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>]</em><em>]</em>) – List of names of the input columns.</p></li>
<li><p><strong>output_columns</strong> (<em>Union</em><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>, </em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#list" title="(in Python v3.8)"><em>list</em></a><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>]</em><em>]</em>) – List of names of the output columns.</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>Dataset, dataset renamed.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># dataset is an instance object of Dataset</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">input_columns</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;input_col1&quot;</span><span class="p">,</span> <span class="s2">&quot;input_col2&quot;</span><span class="p">,</span> <span class="s2">&quot;input_col3&quot;</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">output_columns</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;output_col1&quot;</span><span class="p">,</span> <span class="s2">&quot;output_col2&quot;</span><span class="p">,</span> <span class="s2">&quot;output_col3&quot;</span><span class="p">]</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Create a dataset where input_col1 is renamed to output_col1, and</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># input_col2 is renamed to output_col2, and input_col3 is renamed</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># to output_col3.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">rename</span><span class="p">(</span><span class="n">input_columns</span><span class="o">=</span><span class="n">input_columns</span><span class="p">,</span> <span class="n">output_columns</span><span class="o">=</span><span class="n">output_columns</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.repeat">
<code class="sig-name descname">repeat</code><span class="sig-paren">(</span><em class="sig-param">count=None</em><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.repeat" title="Permalink to this definition">¶</a></dt>
<dd><p>Repeat this dataset <cite>count</cite> times. Repeat infinitely if the count is None or -1.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>The order of using repeat and batch reflects the number of batches. It is recommended that
the repeat operation is used after the batch operation.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>count</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a>) – Number of times the dataset is going to be repeated (default=None).</p>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>Dataset, dataset repeated.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># dataset is an instance object of Dataset</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Create a dataset where the dataset is repeated for 50 epochs</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">repeat</span><span class="p">(</span><span class="mi">50</span><span class="p">)</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Create a dataset where each epoch is shuffled individually</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">shuffle</span><span class="p">(</span><span class="mi">10</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">repeat</span><span class="p">(</span><span class="mi">50</span><span class="p">)</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Create a dataset where the dataset is first repeated for</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># 50 epochs before shuffling. The shuffle operator will treat</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># the entire 50 epochs as one big dataset.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">repeat</span><span class="p">(</span><span class="mi">50</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">shuffle</span><span class="p">(</span><span class="mi">10</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.reset">
<code class="sig-name descname">reset</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.reset" title="Permalink to this definition">¶</a></dt>
<dd><p>Reset the dataset for next epoch.</p>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.save">
<code class="sig-name descname">save</code><span class="sig-paren">(</span><em class="sig-param">file_name</em>, <em class="sig-param">num_files=1</em>, <em class="sig-param">file_type=&quot;mindrecord&quot;</em><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.save" title="Permalink to this definition">¶</a></dt>
<dd><p>Save the dynamic data processed by the dataset pipeline in common dataset format.
Supported dataset formats: ‘mindrecord’ only</p>
<p>Implicit type casting exists when saving data as ‘mindrecord’. The transform table shows how to do type casting.</p>
<table class="colwidths-given docutils align-default" id="id1">
<caption><span class="caption-text">Implicit Type Casting when Saving as ‘mindrecord’</span><a class="headerlink" href="#id1" title="Permalink to this table">¶</a></caption>
<colgroup>
<col style="width: 25%" />
<col style="width: 25%" />
<col style="width: 50%" />
</colgroup>
<thead>
<tr class="row-odd"><th class="head"><p>Type in ‘dataset’</p></th>
<th class="head"><p>Type in ‘mindrecord’</p></th>
<th class="head"><p>Details</p></th>
</tr>
</thead>
<tbody>
<tr class="row-even"><td><p>bool</p></td>
<td><p>None</p></td>
<td><p>Not supported</p></td>
</tr>
<tr class="row-odd"><td><p>int8</p></td>
<td><p>int32</p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>uint8</p></td>
<td><p>bytes(1D uint8)</p></td>
<td><p>Drop dimension</p></td>
</tr>
<tr class="row-odd"><td><p>int16</p></td>
<td><p>int32</p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>uint16</p></td>
<td><p>int32</p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>int32</p></td>
<td><p>int32</p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>uint32</p></td>
<td><p>int64</p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>int64</p></td>
<td><p>int64</p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>uint64</p></td>
<td><p>None</p></td>
<td><p>Not supported</p></td>
</tr>
<tr class="row-odd"><td><p>float16</p></td>
<td><p>float32</p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>float32</p></td>
<td><p>float32</p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>float64</p></td>
<td><p>float64</p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>string</p></td>
<td><p>string</p></td>
<td><p>Multi-dimensional string not supported</p></td>
</tr>
</tbody>
</table>
<div class="admonition note">
<p class="admonition-title">Note</p>
<ol class="arabic simple">
<li><p>To save the samples in order, set dataset’s shuffle to False and num_files to 1.</p></li>
<li><p>Before calling the function, do not use batch operator, repeat operator or data augmentation operators
with random attribute in map operator.</p></li>
<li><p>When array dimension is variable, one-dimensional arrays or
multi-dimensional arrays with variable dimension 0 are supported.</p></li>
<li><p>Mindrecord does not support uint64, multi-dimensional uint8(drop dimension) nor
multi-dimensional string.</p></li>
</ol>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>file_name</strong> (<a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a>) – Path to dataset file.</p></li>
<li><p><strong>num_files</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a><em>, </em><em>optional</em>) – Number of dataset files (default=1).</p></li>
<li><p><strong>file_type</strong> (<a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a><em>, </em><em>optional</em>) – Dataset format (default=’mindrecord’).</p></li>
</ul>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.set_dynamic_columns">
<code class="sig-name descname">set_dynamic_columns</code><span class="sig-paren">(</span><em class="sig-param">columns=None</em><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.set_dynamic_columns" title="Permalink to this definition">¶</a></dt>
<dd><p>Set dynamic shape information of source data, it should be set after the pipeline is defined.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>columns</strong> (<a class="reference external" href="https://docs.python.org/library/stdtypes.html#dict" title="(in Python v3.8)"><em>dict</em></a>) – A dict contains shape information of each column in dataset.
The value of shape[i] is <a class="reference external" href="https://docs.python.org/library/constants.html#None" title="(in Python v3.8)"><code class="xref py py-obj docutils literal notranslate"><span class="pre">None</span></code></a> indicates that the data length of shape[i] is dynamic.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">generator1</span><span class="p">():</span>
<span class="gp">... </span>    <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">100</span><span class="p">):</span>
<span class="gp">... </span>        <span class="k">yield</span> <span class="n">np</span><span class="o">.</span><span class="n">ones</span><span class="p">((</span><span class="mi">16</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="mi">83</span><span class="p">)),</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">i</span><span class="p">)</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">ds</span><span class="o">.</span><span class="n">GeneratorDataset</span><span class="p">(</span><span class="n">generator1</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;data1&quot;</span><span class="p">,</span> <span class="s2">&quot;data2&quot;</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span><span class="o">.</span><span class="n">set_dynamic_columns</span><span class="p">(</span><span class="n">columns</span><span class="o">=</span><span class="p">{</span><span class="s2">&quot;data1&quot;</span><span class="p">:</span> <span class="p">[</span><span class="mi">16</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="mi">83</span><span class="p">],</span> <span class="s2">&quot;data2&quot;</span><span class="p">:</span> <span class="p">[]})</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.shuffle">
<code class="sig-name descname">shuffle</code><span class="sig-paren">(</span><em class="sig-param">buffer_size</em><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.shuffle" title="Permalink to this definition">¶</a></dt>
<dd><p>Randomly shuffles the rows of this dataset using the following policy:</p>
<ol class="arabic simple">
<li><p>Make a shuffle buffer that contains the first buffer_size rows.</p></li>
<li><p>Randomly select an element from the shuffle buffer to be the next row
propagated to the child node.</p></li>
<li><p>Get the next row (if any) from the parent node and put it in the shuffle buffer.</p></li>
<li><p>Repeat steps 2 and 3 until there are no more rows left in the shuffle buffer.</p></li>
</ol>
<p>A random seed can be provided to be used on the first epoch. In every subsequent
epoch, the seed is changed to a new one, randomly generated value.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>buffer_size</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a>) – The size of the buffer (must be larger than 1) for
shuffling. Setting buffer_size equal to the number of rows in the entire
dataset will result in a global shuffle.</p>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>Dataset, dataset shuffled.</p>
</dd>
<dt class="field-odd">Raises</dt>
<dd class="field-odd"><p><a class="reference external" href="https://docs.python.org/library/exceptions.html#RuntimeError" title="(in Python v3.8)"><strong>RuntimeError</strong></a> – If exist sync operators before shuffle.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># dataset is an instance object of Dataset</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Optionally set the seed for the first epoch</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">ds</span><span class="o">.</span><span class="n">config</span><span class="o">.</span><span class="n">set_seed</span><span class="p">(</span><span class="mi">58</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Create a shuffled dataset using a shuffle buffer of size 4</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">shuffle</span><span class="p">(</span><span class="mi">4</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.skip">
<code class="sig-name descname">skip</code><span class="sig-paren">(</span><em class="sig-param">count</em><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.skip" title="Permalink to this definition">¶</a></dt>
<dd><p>Skip the first N elements of this dataset.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>count</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a>) – Number of elements in the dataset to be skipped.</p>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>Dataset, dataset that containing rows like origin rows subtract skipped rows.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># dataset is an instance object of Dataset</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Create a dataset which skips first 3 elements from data</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">skip</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.split">
<code class="sig-name descname">split</code><span class="sig-paren">(</span><em class="sig-param">sizes</em>, <em class="sig-param">randomize=True</em><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.split" title="Permalink to this definition">¶</a></dt>
<dd><p>Split the dataset into smaller, non-overlapping datasets.</p>
<p>This is a general purpose split function which can be called from any operator in the pipeline.
There is another, optimized split function, which will be called automatically if ds.split is
called where ds is a MappableDataset.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>sizes</strong> (<em>Union</em><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#list" title="(in Python v3.8)"><em>list</em></a><em>[</em><a class="reference external" href="https://docs.python.org/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a><em>]</em><em>, </em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#list" title="(in Python v3.8)"><em>list</em></a><em>[</em><a class="reference external" href="https://docs.python.org/library/functions.html#float" title="(in Python v3.8)"><em>float</em></a><em>]</em><em>]</em>) – <p>If a list of integers [s1, s2, …, sn] is
provided, the dataset will be split into n datasets of size s1, size s2, …, size sn
respectively. If the sum of all input sizes does not equal the original dataset size, an
error will throw.
If a list of floats [f1, f2, …, fn] is provided, all floats must be between 0 and 1
and must sum to 1, otherwise an error will throw. The dataset will be split into n
Datasets of size round(f1*K), round(f2*K), …, round(fn*K) where K is the size of the
original dataset.
If after rounding:</p>
<ul>
<li><p>Any size equals 0, an error will occur.</p></li>
<li><p>The sum of split sizes &lt; K, the difference of K - sigma(round(fi * k)) will be added to the first
split.</p></li>
<li><p>The sum of split sizes &gt; K, the difference of sigma(round(fi * K)) - K will be removed from the first
large enough split such that it will have at least 1 row after removing the difference.</p></li>
</ul>
</p></li>
<li><p><strong>randomize</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – Determines whether or not to split the data randomly (default=True).
If True, the data will be randomly split. Otherwise, each split will be created with
consecutive rows from the dataset.</p></li>
</ul>
</dd>
</dl>
<div class="admonition note">
<p class="admonition-title">Note</p>
<ol class="arabic simple">
<li><p>Dataset cannot be sharded if split is going to be called.</p></li>
<li><p>It is strongly recommended to not shuffle the dataset, but use randomize=True instead.
Shuffling the dataset may not be deterministic, which means the data in each split
will be different in each epoch.</p></li>
</ol>
</div>
<dl class="field-list simple">
<dt class="field-odd">Raises</dt>
<dd class="field-odd"><ul class="simple">
<li><p><a class="reference external" href="https://docs.python.org/library/exceptions.html#RuntimeError" title="(in Python v3.8)"><strong>RuntimeError</strong></a> – If get_dataset_size returns None or is not supported for this dataset.</p></li>
<li><p><a class="reference external" href="https://docs.python.org/library/exceptions.html#RuntimeError" title="(in Python v3.8)"><strong>RuntimeError</strong></a> – If <cite>sizes</cite> is list of integers and sum of all elements in sizes does not
    equal the dataset size.</p></li>
<li><p><a class="reference external" href="https://docs.python.org/library/exceptions.html#RuntimeError" title="(in Python v3.8)"><strong>RuntimeError</strong></a> – If <cite>sizes</cite> is list of float and there is a split with size 0 after calculations.</p></li>
<li><p><a class="reference external" href="https://docs.python.org/library/exceptions.html#RuntimeError" title="(in Python v3.8)"><strong>RuntimeError</strong></a> – If the dataset is sharded prior to calling split.</p></li>
<li><p><a class="reference external" href="https://docs.python.org/library/exceptions.html#ValueError" title="(in Python v3.8)"><strong>ValueError</strong></a> – If <cite>sizes</cite> is list of float and not all floats are between 0 and 1, or if the
    floats don’t sum to 1.</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>tuple(Dataset), a tuple of datasets that have been split.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># TextFileDataset is not a mappable dataset, so this non-optimized split will be called.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Since many datasets have shuffle on by default, set shuffle to False if split will be called!</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">ds</span><span class="o">.</span><span class="n">TextFileDataset</span><span class="p">(</span><span class="n">text_file_dataset_dir</span><span class="p">,</span> <span class="n">shuffle</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">train_dataset</span><span class="p">,</span> <span class="n">test_dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">split</span><span class="p">([</span><span class="mf">0.9</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">])</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.sync_update">
<code class="sig-name descname">sync_update</code><span class="sig-paren">(</span><em class="sig-param">condition_name</em>, <em class="sig-param">num_batch=None</em>, <em class="sig-param">data=None</em><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.sync_update" title="Permalink to this definition">¶</a></dt>
<dd><p>Release a blocking condition and trigger callback with given data.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>condition_name</strong> (<a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a>) – The condition name that is used to toggle sending next row.</p></li>
<li><p><strong>num_batch</strong> (<em>Union</em><em>[</em><a class="reference external" href="https://docs.python.org/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a><em>, </em><a class="reference external" href="https://docs.python.org/library/constants.html#None" title="(in Python v3.8)"><em>None</em></a><em>]</em>) – The number of batches (rows) that are released.
When num_batch is None, it will default to the number specified by the
sync_wait operator (default=None).</p></li>
<li><p><strong>data</strong> (<em>Any</em>) – The data passed to the callback, user defined (default=None).</p></li>
</ul>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.sync_wait">
<code class="sig-name descname">sync_wait</code><span class="sig-paren">(</span><em class="sig-param">condition_name</em>, <em class="sig-param">num_batch=1</em>, <em class="sig-param">callback=None</em><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.sync_wait" title="Permalink to this definition">¶</a></dt>
<dd><p>Add a blocking condition to the input Dataset. A synchronize action will be applied.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>condition_name</strong> (<a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a>) – The condition name that is used to toggle sending next row.</p></li>
<li><p><strong>num_batch</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a>) – the number of batches without blocking at the start of each epoch (default=1).</p></li>
<li><p><strong>callback</strong> (<em>function</em>) – The callback function that will be invoked when sync_update is called (default=None).</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>SyncWaitDataset, dataset added a blocking condition.</p>
</dd>
<dt class="field-odd">Raises</dt>
<dd class="field-odd"><p><a class="reference external" href="https://docs.python.org/library/exceptions.html#RuntimeError" title="(in Python v3.8)"><strong>RuntimeError</strong></a> – If condition name already exists.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">gen</span><span class="p">():</span>
<span class="gp">... </span>    <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">100</span><span class="p">):</span>
<span class="gp">... </span>        <span class="k">yield</span> <span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">i</span><span class="p">),)</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">class</span> <span class="nc">Augment</span><span class="p">:</span>
<span class="gp">... </span>    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">loss</span><span class="p">):</span>
<span class="gp">... </span>        <span class="bp">self</span><span class="o">.</span><span class="n">loss</span> <span class="o">=</span> <span class="n">loss</span>
<span class="gp">...</span>
<span class="gp">... </span>    <span class="k">def</span> <span class="nf">preprocess</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">input_</span><span class="p">):</span>
<span class="gp">... </span>        <span class="k">return</span> <span class="n">input_</span>
<span class="gp">...</span>
<span class="gp">... </span>    <span class="k">def</span> <span class="nf">update</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
<span class="gp">... </span>        <span class="bp">self</span><span class="o">.</span><span class="n">loss</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="s2">&quot;loss&quot;</span><span class="p">]</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">batch_size</span> <span class="o">=</span> <span class="mi">4</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">ds</span><span class="o">.</span><span class="n">GeneratorDataset</span><span class="p">(</span><span class="n">gen</span><span class="p">,</span> <span class="n">column_names</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;input&quot;</span><span class="p">])</span>
<span class="go">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">aug</span> <span class="o">=</span> <span class="n">Augment</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">sync_wait</span><span class="p">(</span><span class="n">condition_name</span><span class="o">=</span><span class="s2">&quot;policy&quot;</span><span class="p">,</span> <span class="n">callback</span><span class="o">=</span><span class="n">aug</span><span class="o">.</span><span class="n">update</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">operations</span><span class="o">=</span><span class="p">[</span><span class="n">aug</span><span class="o">.</span><span class="n">preprocess</span><span class="p">],</span> <span class="n">input_columns</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;input&quot;</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">batch</span><span class="p">(</span><span class="n">batch_size</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">count</span> <span class="o">=</span> <span class="mi">0</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">for</span> <span class="n">data</span> <span class="ow">in</span> <span class="n">dataset</span><span class="o">.</span><span class="n">create_dict_iterator</span><span class="p">(</span><span class="n">num_epochs</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">output_numpy</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
<span class="gp">... </span>    <span class="k">assert</span> <span class="n">data</span><span class="p">[</span><span class="s2">&quot;input&quot;</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span> <span class="o">==</span> <span class="n">count</span>
<span class="gp">... </span>    <span class="n">count</span> <span class="o">+=</span> <span class="n">batch_size</span>
<span class="gp">... </span>    <span class="n">data</span> <span class="o">=</span> <span class="p">{</span><span class="s2">&quot;loss&quot;</span><span class="p">:</span> <span class="n">count</span><span class="p">}</span>
<span class="gp">... </span>    <span class="n">dataset</span><span class="o">.</span><span class="n">sync_update</span><span class="p">(</span><span class="n">condition_name</span><span class="o">=</span><span class="s2">&quot;policy&quot;</span><span class="p">,</span> <span class="n">data</span><span class="o">=</span><span class="n">data</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.take">
<code class="sig-name descname">take</code><span class="sig-paren">(</span><em class="sig-param">count=-1</em><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.take" title="Permalink to this definition">¶</a></dt>
<dd><p>Takes at most given numbers of elements from the dataset.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<ol class="arabic simple">
<li><p>If count is greater than the number of elements in the dataset or equal to -1,
all the elements in dataset will be taken.</p></li>
<li><p>The order of using take and batch matters. If take is before batch operation,
then take the given number of rows; otherwise take the given number of batches.</p></li>
</ol>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>count</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#int" title="(in Python v3.8)"><em>int</em></a><em>, </em><em>optional</em>) – Number of elements to be taken from the dataset (default=-1).</p>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>Dataset, dataset taken.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># dataset is an instance object of Dataset</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Create a dataset where the dataset includes 50 elements.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">take</span><span class="p">(</span><span class="mi">50</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.to_device">
<code class="sig-name descname">to_device</code><span class="sig-paren">(</span><em class="sig-param">send_epoch_end=True</em>, <em class="sig-param">create_data_info_queue=False</em><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.to_device" title="Permalink to this definition">¶</a></dt>
<dd><p>Transfer data from CPU to GPU or Ascend or other devices.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>send_epoch_end</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – Whether to send the end of sequence to device or not (default=True).</p></li>
<li><p><strong>create_data_info_queue</strong> (<a class="reference external" href="https://docs.python.org/library/functions.html#bool" title="(in Python v3.8)"><em>bool</em></a><em>, </em><em>optional</em>) – Whether to create queue which stores
types and shapes of data or not(default=False).</p></li>
</ul>
</dd>
</dl>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>If device is Ascend, features of data will be transferred one by one. The limitation
of data transmission per second is 256M.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Returns</dt>
<dd class="field-odd"><p>TransferDataset, dataset for transferring.</p>
</dd>
<dt class="field-even">Raises</dt>
<dd class="field-even"><p><a class="reference external" href="https://docs.python.org/library/exceptions.html#RuntimeError" title="(in Python v3.8)"><strong>RuntimeError</strong></a> – If distribution file path is given but failed to read.</p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.to_json">
<code class="sig-name descname">to_json</code><span class="sig-paren">(</span><em class="sig-param">filename=&quot;&quot;</em><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.to_json" title="Permalink to this definition">¶</a></dt>
<dd><p>Serialize a pipeline into JSON string and dump into file if filename is provided.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>filename</strong> (<a class="reference external" href="https://docs.python.org/library/stdtypes.html#str" title="(in Python v3.8)"><em>str</em></a>) – filename of JSON file to be saved as (default=””).</p>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>str, JSON string of the pipeline.</p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="mindspore.dataset.AGNewsDataset.zip">
<code class="sig-name descname">zip</code><span class="sig-paren">(</span><em class="sig-param">datasets</em><span class="sig-paren">)</span><a class="headerlink" href="#mindspore.dataset.AGNewsDataset.zip" title="Permalink to this definition">¶</a></dt>
<dd><p>Zip the datasets in the sense of input tuple of datasets. Columns in the input datasets must have different
name.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>datasets</strong> (<em>Union</em><em>[</em><a class="reference external" href="https://docs.python.org/library/stdtypes.html#tuple" title="(in Python v3.8)"><em>tuple</em></a><em>, </em><em>class Dataset</em><em>]</em>) – A tuple of datasets or a single class Dataset
to be zipped together with this dataset.</p>
</dd>
<dt class="field-even">Returns</dt>
<dd class="field-even"><p>Dataset, dataset zipped.</p>
</dd>
</dl>
<p class="rubric">Examples</p>
<div class="doctest highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># Create a dataset which is the combination of dataset and dataset_1</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">dataset</span> <span class="o">=</span> <span class="n">dataset</span><span class="o">.</span><span class="n">zip</span><span class="p">(</span><span class="n">dataset_1</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

</dd></dl>

</div>


           </div>
           
          </div>
          <footer>
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
        <a href="mindspore.dataset.AmazonReviewDataset.html" class="btn btn-neutral float-right" title="mindspore.dataset.AmazonReviewDataset" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
        <a href="mindspore.dataset.WIDERFaceDataset.html" class="btn btn-neutral float-left" title="mindspore.dataset.WIDERFaceDataset" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
    </div>

  <hr/>

  <div role="contentinfo">
    <p>
        &#169; Copyright 2021, MindSpore.

    </p>
  </div>
    
    
    
    Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
    
    <a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
    
    provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>
        </div>
      </div>

    </section>

  </div>
  

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  
  
    
   

</body>
</html>